import requests
import random
import json
import time

base = 100000
user_agent_list = [
    'Mozilla/5.0(compatible;MSIE9.0;WindowsNT6.1;Trident/5.0)',
    'Mozilla/4.0(compatible;MSIE8.0;WindowsNT6.0;Trident/4.0)',
    'Mozilla/4.0(compatible;MSIE7.0;WindowsNT6.0)',
    'Opera/9.80(WindowsNT6.1;U;en)Presto/2.8.131Version/11.11',
    'Mozilla/5.0(WindowsNT6.1;rv:2.0.1)Gecko/20100101Firefox/4.0.1',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
    'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
    'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.3.4000 Chrome/30.0.1599.101 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36',
    'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0'
]
# 请求是从哪个链接而来的
referer_list = [
    'https://www.sogou.com/',
    'http://blog.csdn.net/',
    'https://www.baidu.com/',
]
# 随机抽取user_agent和referer设置Headers识别
header = {
    'User-Agent': random.choice(user_agent_list),
    'Referer': random.choice(referer_list)
}
for i in range(6, 17):
    total = 1
    now = 0
    while now < total:
        response = requests.get(
            f"https://weread.qq.com/web/bookListInCategory/{i * base}?maxIndex={now}", headers=header)

        with open(f"data/{i}00.txt", "a+") as fp:
            fp.write(response.text)
            fp.write("\n")
        data = json.loads(response.text)

        if total == 1:
            total = data["totalCount"]
        now += len(data['books'])
        try:
            print(f"{i}--total:{total}--searchIdx:{data['books'][-1]['searchIdx']}")
        except:
            break
        time.sleep(0.1)
